package cn.darksoul3.spark.dataset

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

import scala.beans.BeanProperty

object WCWithDataFrameByRow {

  def main(args: Array[String]): Unit = {

    val ss: SparkSession = SparkSession.builder().appName("WCWithDataSet").master("local[*]").getOrCreate()
    val sc = ss.sparkContext

    val lines: RDD[String] = sc.textFile("C:\\Users\\cary2\\Desktop\\wordcount.txt")

    val words: RDD[Row] = lines.map(line => {
      val fields = line.split("\\s+")
      Row(fields(0))
    })

    val scheme = StructType(List(StructField("name",StringType)))
    val frame = ss.createDataFrame(words, scheme)

    frame.show()
    ss.stop()
  }

}